﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using RCloud.Data;
using System.IO;

namespace RCloud
{
    public partial class Import : System.Web.UI.Page
    {
        protected void Page_Load(object sender, EventArgs e)
        {
            //ImportMovie();
            //ImportTags();
            //SplitRating();
            //ImportRating(822, 900);
            ConvertRating(850);
        }

        #region 文件地址

        const string MovieFile = "~/Data/movies.dat";
        const string RatingFile = "~/Data/ratings.dat";
        const string TagFile = "~/Data/tags.dat";

        #endregion

        #region 导入电影
        /// <summary>
        /// 导入电影数据，包括Genres和Tags
        /// </summary>
        public void ImportMovie()
        {
            //1.定义数据操作类
            MovieUpdateClass muc = new MovieUpdateClass { Mode = Doo.DataClass.UpdateMode.OnlyInsert };
            MovieGenresUpdateClass mgu = new MovieGenresUpdateClass { Mode = Doo.DataClass.UpdateMode.OnlyInsert };
            mgu.UpdateFunc = r => { return true; };

            //2.读取源数据文件
            string[] sourcelines = File.ReadAllLines(Server.MapPath(MovieFile));
            //获取上下文以批量操作
            using (REntities container = muc.GetEntities())
            {
                foreach (string line in sourcelines)
                {
                    //格式为 ID:Title:Genres split by |
                    string[] spinfos = line.Split(new string[] { "::" }, StringSplitOptions.None);
                    int movieID = int.Parse(spinfos[0]);
                    string title = spinfos[1];
                    string[] genres = spinfos[2].Split('|');
                    //已存在的标签
                    Dictionary<string, long> Tags = new Dictionary<string, long>();
                    var dbTags = new Data.List.TagsClass().GetList();
                    Tags = dbTags.ToDictionary(r => r.Tag, r => r.RowID);

                    //写入数据库中
                    //写入Movie
                    muc.ID = movieID;
                    muc.UpdateFunc = r => { r.Title = title; return true; };
                    muc.Process(container, false);
                    //写入Genres
                    mgu.MovieID = movieID;
                    foreach (string g in genres)
                    {
                        mgu.TagID = Tags[g];
                        mgu.Process(container, false);
                    }
                }
                container.SaveChanges();
            }

            //完成了...
        }
        #endregion

        #region 导入标签
        /// <summary>
        /// 导入标签数据
        /// 针对MovieLens数据，由于其评价和标签是分开的。UserID::MovieID::Tag::Timestamp
        /// 考虑到人们虽然随着时间的变化喜好可能变化，但基本认知应该不会变（除了一些excellent的标签），如麻辣、简短、刺激。因此这里暂时不对Timestamp做处理。只简单的把Tag加起来赋予Movie
        /// 在处理评价数据时，再将电影标签用来影响用户
        /// </summary>
        public void ImportTags()
        {
            Dictionary<int, Dictionary<long, int>> MovieTags = new Dictionary<int, Dictionary<long, int>>();
            //已存在的标签
            Dictionary<string, long> Tags = new Dictionary<string, long>();
            var dbTags = new Data.List.TagsClass().GetList();
            Tags = dbTags.ToDictionary(r => r.Tag, r => r.RowID);

            string[] lines = File.ReadAllLines(Server.MapPath(TagFile));

            //对每一行进行处理
            foreach (string l in lines)
            {
                string[] spinfos = l.Split(new string[] { "::" }, StringSplitOptions.None);
                int movieID = int.Parse(spinfos[1]);
                string tag = spinfos[2];

                if (!MovieTags.ContainsKey(movieID)) MovieTags.Add(movieID, new Dictionary<long, int>());
                var tagcount = MovieTags[movieID];

                long tagid = Tags[tag];

                if (!tagcount.ContainsKey(tagid)) tagcount.Add(tagid, 1);
                else tagcount[tagid]++;
            }

            //处理完成，写入初始化数据库
            MovieTagsUpdateClass mtu = new MovieTagsUpdateClass { Mode = Doo.DataClass.UpdateMode.OnlyInsert };
            using (var container = mtu.GetEntities())
            {
                foreach (KeyValuePair<int, Dictionary<long, int>> MT in MovieTags)
                {
                    foreach (KeyValuePair<long, int> MTTags in MT.Value)
                    {
                        mtu.MovieID = MT.Key;
                        mtu.TagID = MTTags.Key;
                        mtu.UpdateFunc = r => { r.SumCount = r.PrevCount = MTTags.Value; return true; };
                        mtu.Process(container, false);
                    }
                }
                container.SaveChanges();
            }
        }
        #endregion

        #region 分割数据按时间段
        /// <summary>
        /// 将原始文件分时间段写入不同的文件
        /// </summary>
        public void SplitRating()
        {
            //太大导入会失败，所以分段导入
            //按时间分段写入不同的文件

            #region 将原始文件分时间段写入不同的文件
            var lines = File.ReadLines(Server.MapPath(RatingFile));
            string path = Server.MapPath("~/Data/");

            Dictionary<int, StreamWriter> files = new Dictionary<int, StreamWriter>();

            foreach (var l in lines)
            {
                string[] spinfos = l.Split(new string[] { "::" }, StringSplitOptions.None);

                int ts = int.Parse(spinfos[3]) / 1000000;
                StreamWriter fs;
                if (!files.TryGetValue(ts, out fs))
                {
                    string file = path + ts + ".txt";
                    fs = File.AppendText(file);
                    files.Add(ts, fs);
                }
                fs.WriteLine(l);
            }

            //关闭所有的文件
            foreach (var f in files.Values)
            {
                f.Close();
            }

            return;
            #endregion

        }
        #endregion

        #region 导入数据

        /// <summary>
        /// 导入评价数据
        /// </summary>
        /// <param name="tickSt">时间段开始，从822开始</param>
        /// <param name="tickEn">时间段结束，到1231结束</param>
        public void ImportRating(int tickSt, int tickEn)
        {
            //导入评价数据与时间有关，目前采用时间分段，即每1W秒作为一个时间点
            //在每个时间点进行计算：将PrevCount*系数+AddCount作为最新的SumCount
            //实验对系数进行取值：0-1（即只要最新的和从不遗忘）
            //由于文件过大，特采用ReadLines而不是ReadAllLines
            //The lines within this file are ordered first by UserID, then, within user, by MovieID. 
            //而我们的流程是按TimeStamp排序，用户和Movie不停的互相影响

            UserRatingHistoryUpdateClass uru = new UserRatingHistoryUpdateClass { Mode = Doo.DataClass.UpdateMode.OnlyInsert };

            const long MaxWaiting = 100000;

            using (var container = uru.GetEntities())
            {
                long waiting = MaxWaiting;

                for (int i = tickSt; i <= tickEn; i++)
                {
                    string fname = Server.MapPath("~/Data/" + i + ".txt");
                    var lines = File.ReadAllLines(fname);
                    foreach (var l in lines)
                    {
                        //UserID::MovieID::Rating::Timestamp(秒)
                        string[] spinfos = l.Split(new string[] { "::" }, StringSplitOptions.None);

                        int uid = int.Parse(spinfos[0]);
                        int mid = int.Parse(spinfos[1]);
                        int rat = (int)(float.Parse(spinfos[2]) * 2);
                        int ts = int.Parse(spinfos[3]);
                        //解析完成

                        uru.UpdateFunc = r =>
                        {
                            r.MovieID = mid;
                            r.UserID = uid;
                            r.Rating = rat;
                            r.TimeStamp = ts;
                            return true;
                        };

                        uru.Process(container, false);
                        waiting--;
                        if (waiting == 0)
                        {
                            container.SaveChanges();
                            waiting = MaxWaiting;
                        }
                    }
                }
                container.SaveChanges();
            }
        }

        #endregion

        #region 处理数据并将其转换成用户偏好、商品标签权重等

        /// <summary>
        /// 处理数据到某个时间点
        /// </summary>
        public void ConvertRating(int tickEn)
        {
            //1.读取所有数据
            Data.List.RatingHistory rhClass = new Data.List.RatingHistory { RowCount = int.MaxValue, TimeStampEnd = tickEn * 1000000 };
            var hisdata = rhClass.GetArray();

            //2.准备电影标签初始数据
            Dictionary<long, UserData> users = new Dictionary<long, UserData>();
            Dictionary<long, MovieData> movies = new Dictionary<long, MovieData>();

            Data.List.MovieGenresClass mtClass = new Data.List.MovieGenresClass { RowCount = int.MaxValue };
            var mvtags = mtClass.GetArray();

            foreach (var mvt in mvtags)
            {
                long mid = mvt.MovieID;
                MovieData md;
                if (!movies.TryGetValue(mid, out md))
                {
                    md = new MovieData();
                    movies.Add(mid, md);
                    md.BeginProcessTag();
                }
                md.ProcessTag(mvt.TagID, 5);
            }

            Data.List.MovieTagClass mtagClass = new Data.List.MovieTagClass { RowCount = int.MaxValue };
            var tags = mtagClass.GetArray();
            foreach (var t in tags)
            {
                long mid = t.MovieID;
                MovieData md;
                if (!movies.TryGetValue(mid, out md))
                {
                    md = new MovieData();
                    movies.Add(mid, md);
                    md.BeginProcessTag();
                }
                md.ProcessTag(t.TagID, 1);
            }

            foreach (MovieData md in movies.Values)
            {
                md.EndProcessTag();
            }

            //3.开始正式处理

            foreach (var his in hisdata)
            {
                MovieData md = movies[his.MovieID];
                UserData ud;
                long uid = his.UserID;
                if (!users.TryGetValue(uid, out ud))
                {
                    ud = new UserData();
                    users.Add(uid, ud);
                }
                int rating = (int)his.Rating;

                //3.1先对用户处理
                ud.RateMovie(md, rating);

                //3.2再对电影处理
                md.BeRated(ud, rating);

            }


            //4.处理完毕，将结果保存到数据库
            Data.MovieTagsNewUpdateClass mtuClass = new MovieTagsNewUpdateClass { Mode = Doo.DataClass.UpdateMode.OnlyInsert };
            using (var container = mtuClass.GetEntities())
            {
                foreach (KeyValuePair<long, MovieData> kv in movies)
                {
                    mtuClass.MovieID = kv.Key;
                    foreach (KeyValuePair<long, PItem> rate in kv.Value)
                    {
                        mtuClass.UpdateFunc = r => { r.SumCount = rate.Value.SumCount; r.TagID = rate.Value.TagID; return true; };
                        mtuClass.Process(container, false);
                    }
                    container.SaveChanges();
                }
            }
        }

        #endregion

        class PData : Dictionary<long, PItem>
        {
            int Max = 1;

            /// <summary>
            /// 及格线
            /// </summary>
            int Avg = 0;

            /// <summary>
            /// 已缓存的有效偏好
            /// </summary>
            Dictionary<long, int> cachedTags = new Dictionary<long, int>();

            /// <summary>
            /// 取出已概化为1-5分的有效偏好
            /// </summary>
            /// <returns></returns>
            public Dictionary<long, int> TopTags
            {
                get { return cachedTags; }
            }

            bool needRefresh = false;

            /// <summary>
            /// 准备工作
            /// </summary>
            public void BeginProcessTag()
            {
                //暂时没有
            }

            /// <summary>
            /// 做完了ProcessTag需要做一些收尾工作
            /// </summary>
            public void EndProcessTag()
            {
                if (needRefresh)
                {
                    foreach (KeyValuePair<long, int> kv in cachedTags)
                    {
                        if (kv.Value < Avg)
                        {
                            cachedTags.Remove(kv.Key);
                            continue;
                        }

                        int fin = (int)((this[kv.Key].SumCount - Avg) / Max) + 1;
                        if (fin == kv.Value) continue;
                        cachedTags[kv.Key] = fin;
                    }
                    needRefresh = false;
                }
            }

            public void ProcessTag(long tagID, int score)
            {
                PItem pi;
                if (TryGetValue(tagID, out pi))
                {
                    int r = pi.SumCount + score;
                    if (r > Max)
                    {
                        r = Max;
                        Avg = (int)(Max * 0.6);
                        //Max被改变，要修改cachedTags中所有数据了
                        needRefresh = true;

                    }
                    else if (r > Avg && !cachedTags.ContainsKey(tagID))
                    {
                        cachedTags.Add(tagID, (int)((r - Avg) / Max) + 1);
                    }
                    pi.SumCount = r;
                }
                else
                {
                    pi = new PItem { TagID = tagID, SumCount = score };
                    Add(tagID, pi);
                    if (score > Avg)
                    {
                        cachedTags.Add(tagID, (int)((score - Avg) / Max) + 1);
                    }
                }
            }

        }

        /// <summary>
        /// 用户数据
        /// </summary>
        class UserData : PData
        {

            /// <summary>
            /// 为某个影片做了投票，更新自己的偏好数据库
            /// </summary>
            /// <param name="mv"></param>
            public void RateMovie(MovieData mv, int rate)
            {
                //取出影片中有效偏好，对自己的数据进行修正
                var mvtags = mv.TopTags;
                BeginProcessTag();
                foreach (KeyValuePair<long, int> kv in mvtags)
                {
                    ProcessTag(kv.Key, kv.Value);
                }
                EndProcessTag();
            }
        }

        class MovieData : PData
        {
            /// <summary>
            /// 被人投票，更新自己的偏好数据库
            /// </summary>
            /// <param name="mv"></param>
            public void BeRated(UserData ud, int rate)
            {
                //取出影片中有效偏好，对自己的数据进行修正
                var mvtags = ud.TopTags;
                BeginProcessTag();
                foreach (KeyValuePair<long, int> kv in mvtags)
                {
                    ProcessTag(kv.Key, kv.Value);
                }
                EndProcessTag();
            }
        }

        struct PItem
        {
            /// <summary>
            /// 标签
            /// </summary>
            public long TagID;
            /// <summary>
            /// 计数
            /// </summary>
            public int SumCount;
            /// <summary>
            /// 前次计数
            /// </summary>
            public int PrevCount;
        }
    }
}